<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=9"/>
<title>JSpider Library: src/main/java/com/spider/jspiderlibrary2/Spider.java Source File</title>

<link href="tabs.css" rel="stylesheet" type="text/css"/>
<link href="doxygen.css" rel="stylesheet" type="text/css" />

<link href="search/search.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="jquery.js"></script>
<script type="text/javascript" src="search/search.js"></script>
<script type="text/javascript">
  $(document).ready(function() { searchBox.OnSelectItem(0); });
</script>

</head>
<body>
<div id="top"><!-- do not remove this div! -->


<div id="titlearea">
<table cellspacing="0" cellpadding="0">
 <tbody>
 <tr style="height: 56px;">
  
  
  <td style="padding-left: 0.5em;">
   <div id="projectname">JSpider Library
   
   </div>
   
  </td>
  
  
  
 </tr>
 </tbody>
</table>
</div>

<!-- Generated by Doxygen 1.7.6.1 -->
<script type="text/javascript">
var searchBox = new SearchBox("searchBox", "search",false,'Search');
</script>
  <div id="navrow1" class="tabs">
    <ul class="tablist">
      <li><a href="index.html"><span>Main&#160;Page</span></a></li>
      <li><a href="namespaces.html"><span>Packages</span></a></li>
      <li><a href="annotated.html"><span>Classes</span></a></li>
      <li class="current"><a href="files.html"><span>Files</span></a></li>
      <li>
        <div id="MSearchBox" class="MSearchBoxInactive">
        <span class="left">
          <img id="MSearchSelect" src="search/mag_sel.png"
               onmouseover="return searchBox.OnSearchSelectShow()"
               onmouseout="return searchBox.OnSearchSelectHide()"
               alt=""/>
          <input type="text" id="MSearchField" value="Search" accesskey="S"
               onfocus="searchBox.OnSearchFieldFocus(true)" 
               onblur="searchBox.OnSearchFieldFocus(false)" 
               onkeyup="searchBox.OnSearchFieldChange(event)"/>
          </span><span class="right">
            <a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><img id="MSearchCloseImg" border="0" src="search/close.png" alt=""/></a>
          </span>
        </div>
      </li>
    </ul>
  </div>
  <div id="navrow2" class="tabs2">
    <ul class="tablist">
      <li><a href="files.html"><span>File&#160;List</span></a></li>
    </ul>
  </div>
</div>
<div class="header">
  <div class="headertitle">
<div class="title">src/main/java/com/spider/jspiderlibrary2/Spider.java</div>  </div>
</div><!--header-->
<div class="contents">
<a href="_spider_8java.html">Go to the documentation of this file.</a><div class="fragment"><pre class="fragment"><a name="l00001"></a>00001 <span class="keyword">package </span>com.spider.jspiderlibrary2;
<a name="l00002"></a>00002 
<a name="l00003"></a>00003 <span class="keyword">import</span> java.util.*;
<a name="l00004"></a>00004 <span class="keyword">import</span> java.net.*;
<a name="l00005"></a>00005 <span class="keyword">import</span> java.io.*;
<a name="l00006"></a>00006 <span class="keyword">import</span> javax.swing.text.*;
<a name="l00007"></a>00007 <span class="keyword">import</span> javax.swing.text.html.*;
<a name="l00008"></a>00008 <span class="keyword">import</span> java.util.regex.Pattern;
<a name="l00020"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html">00020</a> <span class="keyword">public</span> <span class="keyword">class </span><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html">Spider</a> {
<a name="l00021"></a>00021 
<a name="l00022"></a>00022     <span class="keyword">private</span> <span class="keyword">final</span> <span class="keyword">static</span> Pattern BOTTOMFILTERS =
<a name="l00023"></a>00023             Pattern.compile(<span class="stringliteral">&quot;.*(\\.(css|js|bmp|gif|jpe?g&quot;</span> + <span class="stringliteral">&quot;|png|tiff?|mid|mp2|mp3|ppt|ico|JPG|mp4&quot;</span> + <span class="stringliteral">&quot;|wav|avi|mov|mpeg|ram|m4v|pdf&quot;</span> + <span class="stringliteral">&quot;|rm|smil|wmv|swf|wma|zip|rar|gz|xml|php\\?rsd))$&quot;</span>);
<a name="l00024"></a>00024     <span class="keyword">private</span> <span class="keyword">final</span> <span class="keyword">static</span> Pattern MIDDLEFILTERS =
<a name="l00025"></a>00025             Pattern.compile(<span class="stringliteral">&quot;.*(rss|css).*&quot;</span>);
<a name="l00029"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#aa28c7efd1d89fd2b553833f289946477">00029</a>     <span class="keyword">protected</span> ArrayList <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#aa28c7efd1d89fd2b553833f289946477">workloadError</a> = <span class="keyword">new</span> ArrayList(3);
<a name="l00034"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a7ab086f00e291c450e31bac92b3b1f3a">00034</a>     <span class="keyword">protected</span> ArrayList <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a7ab086f00e291c450e31bac92b3b1f3a">workloadWaiting</a> = <span class="keyword">new</span> ArrayList(3);
<a name="l00035"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#ad9e3dbf06a7d4a976fadc85df5fcf81e">00035</a>     <span class="keyword">protected</span> ArrayList <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#ad9e3dbf06a7d4a976fadc85df5fcf81e">workloadProcessed</a> = <span class="keyword">new</span> ArrayList(3);
<a name="l00040"></a>00040     <span class="keyword">private</span> StringBuilder SBConcatText = <span class="keyword">new</span> StringBuilder();
<a name="l00045"></a>00045     <span class="keyword">private</span> HashMap HashMapPage = <span class="keyword">new</span> HashMap();
<a name="l00055"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a234dbec15186b1bc7750700697ef233b">00055</a>     <span class="keyword">protected</span> <a class="code" href="interfacecom_1_1spider_1_1jspiderlibrary2_1_1_i_spider_reportable.html">ISpiderReportable</a> <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a234dbec15186b1bc7750700697ef233b">report</a>;
<a name="l00060"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a2dbab52498bf5a06ce0344116aed3b3b">00060</a>     <span class="keyword">protected</span> <span class="keywordtype">boolean</span> <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a9053f35d953c1311c1c98710561b49e4">cancel</a> = <span class="keyword">false</span>;
<a name="l00061"></a>00061     <span class="keyword">private</span> <span class="keywordtype">boolean</span> permisible = <span class="keyword">false</span>;
<a name="l00062"></a>00062 
<a name="l00070"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a1522b55496fac7640ea54b6623d1072f">00070</a>     <span class="keyword">public</span> <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a1522b55496fac7640ea54b6623d1072f">Spider</a>(<a class="code" href="interfacecom_1_1spider_1_1jspiderlibrary2_1_1_i_spider_reportable.html">ISpiderReportable</a> <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a234dbec15186b1bc7750700697ef233b">report</a>) {
<a name="l00071"></a>00071         this.report = <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a234dbec15186b1bc7750700697ef233b">report</a>;
<a name="l00072"></a>00072     }
<a name="l00073"></a>00073 
<a name="l00079"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#aa2331130fc2e5be2a54b120da24bc348">00079</a>     <span class="keyword">public</span> Collection <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#aa2331130fc2e5be2a54b120da24bc348">getWorkloadError</a>() {
<a name="l00080"></a>00080         <span class="keywordflow">return</span> <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#aa28c7efd1d89fd2b553833f289946477">workloadError</a>;
<a name="l00081"></a>00081     }
<a name="l00082"></a>00082 
<a name="l00087"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a77c4d59ea95fc8c7661b87bbf5da4f86">00087</a>     <span class="keyword">public</span> Collection <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a77c4d59ea95fc8c7661b87bbf5da4f86">getWorkloadWaiting</a>() {
<a name="l00088"></a>00088         <span class="keywordflow">return</span> <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a7ab086f00e291c450e31bac92b3b1f3a">workloadWaiting</a>;
<a name="l00089"></a>00089     }
<a name="l00090"></a>00090 
<a name="l00095"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a6dca4a5d46dcdf9b97bc9cb799ba2c9c">00095</a>     <span class="keyword">public</span> Collection <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a6dca4a5d46dcdf9b97bc9cb799ba2c9c">getWorkloadProcessed</a>() {
<a name="l00096"></a>00096         <span class="keywordflow">return</span> <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#ad9e3dbf06a7d4a976fadc85df5fcf81e">workloadProcessed</a>;
<a name="l00097"></a>00097     }
<a name="l00098"></a>00098 
<a name="l00102"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#ae38b2465861c53a4dac6920a32daa2a2">00102</a>     <span class="keyword">public</span> <span class="keywordtype">void</span> <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#ae38b2465861c53a4dac6920a32daa2a2">clear</a>() {
<a name="l00103"></a>00103         <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#aa2331130fc2e5be2a54b120da24bc348">getWorkloadError</a>().clear();
<a name="l00104"></a>00104         <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a77c4d59ea95fc8c7661b87bbf5da4f86">getWorkloadWaiting</a>().clear();
<a name="l00105"></a>00105         <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a6dca4a5d46dcdf9b97bc9cb799ba2c9c">getWorkloadProcessed</a>().clear();
<a name="l00106"></a>00106         <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a472cd53008e1a952f2d8dcfe06f6e744">clearHashMapPages</a>();
<a name="l00107"></a>00107     }
<a name="l00108"></a>00108 
<a name="l00113"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a9053f35d953c1311c1c98710561b49e4">00113</a>     <span class="keyword">public</span> <span class="keywordtype">void</span> <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a9053f35d953c1311c1c98710561b49e4">cancel</a>() {
<a name="l00114"></a>00114         <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a9053f35d953c1311c1c98710561b49e4">cancel</a> = <span class="keyword">true</span>;
<a name="l00115"></a>00115     }
<a name="l00116"></a>00116 
<a name="l00122"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a6c7061bb6b58bdbe6be1a162cf470c85">00122</a>     <span class="keyword">public</span> <span class="keywordtype">void</span> <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a6c7061bb6b58bdbe6be1a162cf470c85">addURL</a>(URL url) {
<a name="l00123"></a>00123 
<a name="l00124"></a>00124         <span class="keywordflow">if</span> (url.toString().endsWith(<span class="stringliteral">&quot;/&quot;</span>)) {
<a name="l00125"></a>00125             String splitslash = url.toString().substring(0, url.toString().length() - 1);
<a name="l00126"></a>00126             <span class="comment">//Quito el Slash al final del url si existiera</span>
<a name="l00127"></a>00127             <span class="keywordflow">try</span> {
<a name="l00128"></a>00128                 url = <span class="keyword">new</span> URL(splitslash);
<a name="l00129"></a>00129             } <span class="comment">//A partir de esa string creo otra url nueva sin la barra para evitar</span>
<a name="l00130"></a>00130             <span class="comment">//ambiguedades</span>
<a name="l00131"></a>00131             <span class="keywordflow">catch</span> (MalformedURLException e) {
<a name="l00132"></a>00132                 <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a953b39d3048cc57e3b5a6050a52931fa">log</a>(<span class="stringliteral">&quot;malformed URL: &quot;</span> + splitslash );
<a name="l00133"></a>00133             }
<a name="l00134"></a>00134         }
<a name="l00135"></a>00135         <span class="comment">//Si ya esta presente la url en alguna de las 3 listas no la empleo mas</span>
<a name="l00136"></a>00136         <span class="keywordflow">if</span> (<a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a77c4d59ea95fc8c7661b87bbf5da4f86">getWorkloadWaiting</a>().contains(url)) {
<a name="l00137"></a>00137             <span class="keywordflow">return</span>;
<a name="l00138"></a>00138         }
<a name="l00139"></a>00139         <span class="keywordflow">if</span> (<a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#aa2331130fc2e5be2a54b120da24bc348">getWorkloadError</a>().contains(url)) {
<a name="l00140"></a>00140             <span class="keywordflow">return</span>;
<a name="l00141"></a>00141         }
<a name="l00142"></a>00142         <span class="keywordflow">if</span> (<a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a6dca4a5d46dcdf9b97bc9cb799ba2c9c">getWorkloadProcessed</a>().contains(url)) {
<a name="l00143"></a>00143             <span class="keywordflow">return</span>;
<a name="l00144"></a>00144         }
<a name="l00145"></a>00145         <span class="keywordflow">if</span> (!isTextPage(url)) {
<a name="l00146"></a>00146             <span class="keywordflow">return</span>;
<a name="l00147"></a>00147         }
<a name="l00148"></a>00148         <span class="comment">//Sino la añado al cola de espera</span>
<a name="l00149"></a>00149         <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a77c4d59ea95fc8c7661b87bbf5da4f86">getWorkloadWaiting</a>().add(url);
<a name="l00150"></a>00150     }
<a name="l00151"></a>00151 
<a name="l00157"></a>00157     <span class="keyword">private</span> <span class="keywordtype">boolean</span> isTextPage(URLConnection connection) {
<a name="l00158"></a>00158         <span class="keywordflow">if</span> (connection == null) {
<a name="l00159"></a>00159             <span class="keywordflow">return</span> <span class="keyword">false</span>;
<a name="l00160"></a>00160         }
<a name="l00161"></a>00161         <span class="keywordflow">return</span> <span class="keyword">true</span>;
<a name="l00162"></a>00162     }
<a name="l00163"></a>00163 
<a name="l00164"></a>00164     <span class="keyword">private</span> <span class="keywordtype">boolean</span> isTextPage(URL url) {
<a name="l00165"></a>00165         <span class="keywordflow">try</span> {
<a name="l00166"></a>00166             URLConnection connection = url.openConnection();
<a name="l00167"></a>00167             <span class="keywordflow">return</span> isTextPage(connection);
<a name="l00168"></a>00168         } <span class="keywordflow">catch</span> (IOException e) {
<a name="l00169"></a>00169             <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a953b39d3048cc57e3b5a6050a52931fa">log</a>(<span class="stringliteral">&quot;Errorurl, &quot;</span> + e.getMessage());
<a name="l00170"></a>00170         }
<a name="l00171"></a>00171         <span class="keywordflow">return</span> <span class="keyword">false</span>;
<a name="l00172"></a>00172     }
<a name="l00173"></a>00173 
<a name="l00178"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#aff157f7c46f1ba056dac08797299e0f5">00178</a>     <span class="keyword">public</span> <span class="keywordtype">void</span> <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#aff157f7c46f1ba056dac08797299e0f5">processURL</a>(URL url) {
<a name="l00179"></a>00179         <span class="keywordflow">try</span> {
<a name="l00180"></a>00180             <span class="comment">//log(&quot;Processing: &quot; + url );</span>
<a name="l00181"></a>00181             <span class="keywordflow">if</span> (!<a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_robots_parser.html">RobotsParser</a>.<a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_robots_parser.html#a763eb021063080b7cbfc3a0477b89b38">robotSafe</a>(url)) {
<a name="l00182"></a>00182                 <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a77c4d59ea95fc8c7661b87bbf5da4f86">getWorkloadWaiting</a>().remove(url);
<a name="l00183"></a>00183                 <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a6dca4a5d46dcdf9b97bc9cb799ba2c9c">getWorkloadProcessed</a>().add(url);
<a name="l00184"></a>00184                 <span class="keywordflow">return</span>;
<a name="l00185"></a>00185             }
<a name="l00186"></a>00186             <span class="comment">//verifica si la url esta en el archivo</span>
<a name="l00187"></a>00187             <span class="comment">//robots para ver si puede entrar, sino se puede entrar</span>
<a name="l00188"></a>00188             <span class="comment">//retorno terminando el metodo</span>
<a name="l00189"></a>00189             <span class="comment">//Trae el contenidos de la url</span>
<a name="l00190"></a>00190             URLConnection connection = url.openConnection();
<a name="l00191"></a>00191             <span class="keywordflow">if</span> (!isTextPage(connection)) {
<a name="l00192"></a>00192                 <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a77c4d59ea95fc8c7661b87bbf5da4f86">getWorkloadWaiting</a>().remove(url);
<a name="l00193"></a>00193                 <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a6dca4a5d46dcdf9b97bc9cb799ba2c9c">getWorkloadProcessed</a>().add(url);
<a name="l00194"></a>00194                 <span class="keywordflow">return</span>;
<a name="l00195"></a>00195             }
<a name="l00196"></a>00196             <span class="comment">//Creo los flujos y reader para leer y parsear la url</span>
<a name="l00197"></a>00197             InputStream inputStreamURL = connection.getInputStream();
<a name="l00198"></a>00198             Reader readerURL = <span class="keyword">new</span> InputStreamReader(inputStreamURL);
<a name="l00199"></a>00199             <span class="comment">//Parseo la URL usando el HTML parser</span>
<a name="l00200"></a>00200             HTMLEditorKit.Parser parse = <span class="keyword">new</span> <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_h_t_m_l_parse.html">HTMLParse</a>().<a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_h_t_m_l_parse.html#a490841fe99ac950af2fe545785edadae">getParser</a>();
<a name="l00201"></a>00201             parse.parse(readerURL, <span class="keyword">new</span> <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider_1_1_parser.html">Parser</a>(url), <span class="keyword">true</span>);
<a name="l00202"></a>00202         } <span class="keywordflow">catch</span> (IOException e) {
<a name="l00203"></a>00203             <span class="comment">//Cuando termino de leer la URL fuerzo a que se dispare una excepcion</span>
<a name="l00204"></a>00204             <span class="comment">//E informo error en la URL</span>
<a name="l00205"></a>00205             <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a77c4d59ea95fc8c7661b87bbf5da4f86">getWorkloadWaiting</a>().remove(url);
<a name="l00206"></a>00206             <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#aa2331130fc2e5be2a54b120da24bc348">getWorkloadError</a>().add(url);
<a name="l00207"></a>00207             <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a234dbec15186b1bc7750700697ef233b">report</a>.<a class="code" href="interfacecom_1_1spider_1_1jspiderlibrary2_1_1_i_spider_reportable.html#a73e089019826aba4f610eaa4974e6ad8">spiderFoundURLError</a>(url);
<a name="l00208"></a>00208             <span class="keywordflow">return</span>;
<a name="l00209"></a>00209         }
<a name="l00210"></a>00210         <span class="comment">//Quito la URL actual de waiting ya que fue procesada</span>
<a name="l00211"></a>00211         <span class="comment">//Añado hash map la url actual, con todo el texto de esa url</span>
<a name="l00212"></a>00212         <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a77c4d59ea95fc8c7661b87bbf5da4f86">getWorkloadWaiting</a>().remove(url);
<a name="l00213"></a>00213         HashMapPage.put(url, SBConcatText.toString());
<a name="l00214"></a>00214         <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a6dca4a5d46dcdf9b97bc9cb799ba2c9c">getWorkloadProcessed</a>().add(url);
<a name="l00215"></a>00215         <span class="comment">//Vacio el StringBuilder</span>
<a name="l00216"></a>00216         SBConcatText.delete(0, SBConcatText.length());
<a name="l00217"></a>00217         System.out.println(<span class="stringliteral">&quot;EL tamao de waiting&quot;</span> + <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a77c4d59ea95fc8c7661b87bbf5da4f86">getWorkloadWaiting</a>().size() + <span class="stringliteral">&quot;EL tamaño de processesed&quot;</span> + <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a6dca4a5d46dcdf9b97bc9cb799ba2c9c">getWorkloadProcessed</a>().size());
<a name="l00218"></a>00218         <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_communicator.html">Communicator</a>.<a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_communicator.html#af85276b4b0e742776669cd6af974480c">totalLinks</a>++;
<a name="l00219"></a>00219     }
<a name="l00220"></a>00220 
<a name="l00221"></a>00221     <span class="comment">//Traer el hashMap</span>
<a name="l00222"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#adbf2d01cc0ea713a8c4a8ea9c1cbaf56">00222</a>     <span class="keyword">public</span> HashMap <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#adbf2d01cc0ea713a8c4a8ea9c1cbaf56">getHashMapPages</a>() {
<a name="l00223"></a>00223         <span class="keywordflow">return</span> HashMapPage;
<a name="l00224"></a>00224     }
<a name="l00225"></a>00225 
<a name="l00226"></a>00226     <span class="comment">//Borrar El hashMap</span>
<a name="l00227"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a472cd53008e1a952f2d8dcfe06f6e744">00227</a>     <span class="keyword">public</span> <span class="keywordtype">void</span> <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a472cd53008e1a952f2d8dcfe06f6e744">clearHashMapPages</a>() {
<a name="l00228"></a>00228         HashMapPage.clear();
<a name="l00229"></a>00229     }
<a name="l00230"></a>00230 
<a name="l00234"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#af82be12bad4bc1e88f494645106ac7e0">00234</a>     <span class="keyword">public</span> <span class="keywordtype">void</span> <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#af82be12bad4bc1e88f494645106ac7e0">begin</a>() {
<a name="l00235"></a>00235         <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a9053f35d953c1311c1c98710561b49e4">cancel</a> = <span class="keyword">false</span>;
<a name="l00236"></a>00236         <span class="keywordflow">while</span> (!<a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a77c4d59ea95fc8c7661b87bbf5da4f86">getWorkloadWaiting</a>().isEmpty() &amp;&amp; !<a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a9053f35d953c1311c1c98710561b49e4">cancel</a>) {
<a name="l00237"></a>00237             Object list[] = <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a77c4d59ea95fc8c7661b87bbf5da4f86">getWorkloadWaiting</a>().toArray();
<a name="l00238"></a>00238             <span class="comment">//Lo convierto en un array de object para procesar las URL</span>
<a name="l00239"></a>00239             <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; (i &lt; list.length) &amp;&amp; !<a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a9053f35d953c1311c1c98710561b49e4">cancel</a>; i++) {
<a name="l00240"></a>00240                 <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#aff157f7c46f1ba056dac08797299e0f5">processURL</a>((URL) list[i]);
<a name="l00241"></a>00241                 <span class="keywordflow">if</span> (<a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a6dca4a5d46dcdf9b97bc9cb799ba2c9c">getWorkloadProcessed</a>().size() == 5000) {
<a name="l00242"></a>00242                     this.<a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a77c4d59ea95fc8c7661b87bbf5da4f86">getWorkloadWaiting</a>().clear();
<a name="l00243"></a>00243                     <span class="keywordflow">break</span>;
<a name="l00244"></a>00244                 }
<a name="l00245"></a>00245             }
<a name="l00246"></a>00246         }
<a name="l00247"></a>00247     }
<a name="l00248"></a>00248 
<a name="l00253"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider_1_1_parser.html">00253</a>     <span class="keyword">protected</span> <span class="keyword">class </span><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider_1_1_parser.html">Parser</a>
<a name="l00254"></a>00254             <span class="keyword">extends</span> HTMLEditorKit.ParserCallback {
<a name="l00255"></a>00255         <span class="comment">//URL base del sitio web donde empieza el crawleo</span>
<a name="l00256"></a>00256 
<a name="l00257"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider_1_1_parser.html#aaac5fcd0aa0bdd67e4c6842d2657231d">00257</a>         <span class="keyword">protected</span> URL <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider_1_1_parser.html#aaac5fcd0aa0bdd67e4c6842d2657231d">base</a>;
<a name="l00258"></a>00258         <span class="comment">//Verifico que si el tag es indeseado o no</span>
<a name="l00259"></a>00259         <span class="keyword">private</span> <span class="keywordtype">boolean</span> isUndesiredTag = <span class="keyword">false</span>;
<a name="l00260"></a>00260 
<a name="l00261"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider_1_1_parser.html#a72ae3a543c9a4538a39e552b7118bd6d">00261</a>         <span class="keyword">public</span> <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider_1_1_parser.html#a72ae3a543c9a4538a39e552b7118bd6d">Parser</a>(URL <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider_1_1_parser.html#aaac5fcd0aa0bdd67e4c6842d2657231d">base</a>) {
<a name="l00262"></a>00262             this.base = <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider_1_1_parser.html#aaac5fcd0aa0bdd67e4c6842d2657231d">base</a>;
<a name="l00263"></a>00263         }
<a name="l00264"></a>00264 
<a name="l00272"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider_1_1_parser.html#a04eb14f6a2105866c962a0b1bdc9c568">00272</a>         <span class="keyword">public</span> <span class="keywordtype">void</span> <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider_1_1_parser.html#a04eb14f6a2105866c962a0b1bdc9c568">tagHandler</a>(HTML.Tag tag) {
<a name="l00273"></a>00273             Collection&lt;String&gt; undesiredTags = <span class="keyword">new</span> ArrayList&lt;String&gt;();
<a name="l00274"></a>00274             <span class="comment">//Creo una collecion de los tags indeseados</span>
<a name="l00275"></a>00275             <span class="comment">//Y luego verifico si el tag actual esta dentro de los indeseados</span>
<a name="l00276"></a>00276             undesiredTags.add(HTML.Tag.OPTION.toString());
<a name="l00277"></a>00277             undesiredTags.add(HTML.Tag.LI.toString());
<a name="l00278"></a>00278             undesiredTags.add(HTML.Tag.SCRIPT.toString());
<a name="l00279"></a>00279             undesiredTags.add(<span class="stringliteral">&quot;time&quot;</span>);
<a name="l00280"></a>00280             undesiredTags.add(HTML.Tag.STYLE.toString());
<a name="l00281"></a>00281             undesiredTags.add(HTML.Tag.CODE.toString());
<a name="l00282"></a>00282             undesiredTags.add(<span class="stringliteral">&quot;embed&quot;</span>);
<a name="l00283"></a>00283             undesiredTags.add(HTML.Tag.SPAN.toString());
<a name="l00284"></a>00284             <span class="keywordflow">for</span> (String tag2 : undesiredTags) {
<a name="l00285"></a>00285                 <span class="keywordflow">if</span> (tag2.equalsIgnoreCase(tag.toString())) {
<a name="l00286"></a>00286                     isUndesiredTag = <span class="keyword">true</span>;
<a name="l00287"></a>00287                     <span class="keywordflow">break</span>;
<a name="l00288"></a>00288                 }
<a name="l00289"></a>00289             }
<a name="l00290"></a>00290             <span class="keywordflow">return</span>;
<a name="l00291"></a>00291         }
<a name="l00292"></a>00292 
<a name="l00293"></a>00293         <span class="comment">//Recorre  la url en busca de links a otras pag con href</span>
<a name="l00294"></a>00294         @Override
<a name="l00295"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider_1_1_parser.html#ae14a442ac7232170e7db1e8f872d6f8a">00295</a>         <span class="keyword">public</span> <span class="keywordtype">void</span> <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider_1_1_parser.html#ae14a442ac7232170e7db1e8f872d6f8a">handleSimpleTag</a>(HTML.Tag tag,
<a name="l00296"></a>00296                 MutableAttributeSet atributeSet, <span class="keywordtype">int</span> pos) {
<a name="l00297"></a>00297             <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider_1_1_parser.html#a04eb14f6a2105866c962a0b1bdc9c568">tagHandler</a>(tag);
<a name="l00298"></a>00298             String href = (String) atributeSet.getAttribute(HTML.Attribute.HREF);
<a name="l00299"></a>00299             <span class="keywordflow">if</span> ((href != null) &amp;&amp; ((BOTTOMFILTERS.matcher(href).matches()) || (MIDDLEFILTERS.matcher(href).matches()))) {
<a name="l00300"></a>00300                 <span class="comment">//Verifco si el link encontrado tiene extension valida de texto</span>
<a name="l00301"></a>00301                 <span class="comment">//Html para luego poder acceder a ese contenido</span>
<a name="l00302"></a>00302                 <span class="keywordflow">return</span>;
<a name="l00303"></a>00303             }
<a name="l00304"></a>00304             <span class="keywordflow">if</span> (isTagText(tag.toString())) {
<a name="l00305"></a>00305                 permisible = <span class="keyword">true</span>;
<a name="l00306"></a>00306             }
<a name="l00307"></a>00307             <span class="keywordflow">if</span> ((href == null) &amp;&amp; (tag == HTML.Tag.FRAME)) {
<a name="l00308"></a>00308                 href = (String) atributeSet.getAttribute(HTML.Attribute.SRC);
<a name="l00309"></a>00309             }
<a name="l00310"></a>00310             <span class="comment">//busca href dentro contenedor de framset</span>
<a name="l00311"></a>00311             <span class="comment">//mas info en http://www.w3schools.com/tags/tag_frame.asp</span>
<a name="l00312"></a>00312             <span class="keywordflow">if</span> (href == null) {
<a name="l00313"></a>00313                 <span class="keywordflow">return</span>;
<a name="l00314"></a>00314             }
<a name="l00315"></a>00315             <span class="comment">//Genero un substring de los links que contienen #</span>
<a name="l00316"></a>00316             <span class="keywordtype">int</span> i = href.indexOf(<span class="charliteral">&#39;#&#39;</span>);
<a name="l00317"></a>00317             <span class="keywordflow">if</span> (i != -1) {
<a name="l00318"></a>00318                 href = href.substring(0, i);
<a name="l00319"></a>00319             }
<a name="l00320"></a>00320             <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider_1_1_parser.html#aa7bd808ea40da16b1e666113db1b1616">handleLink</a>(<a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider_1_1_parser.html#aaac5fcd0aa0bdd67e4c6842d2657231d">base</a>, href);
<a name="l00321"></a>00321         }
<a name="l00322"></a>00322 
<a name="l00323"></a>00323         <span class="keyword">private</span> <span class="keywordtype">boolean</span> isTagText(String tag) {
<a name="l00324"></a>00324             <span class="comment">//Links deseados a recorrer que contienen texto de html util</span>
<a name="l00325"></a>00325             Collection&lt;String&gt; desiredTags = <span class="keyword">new</span> ArrayList&lt;String&gt;();
<a name="l00326"></a>00326             desiredTags.add(HTML.Tag.B.toString());
<a name="l00327"></a>00327             desiredTags.add(HTML.Tag.ADDRESS.toString());
<a name="l00328"></a>00328             desiredTags.add(HTML.Tag.BIG.toString());
<a name="l00329"></a>00329             desiredTags.add(HTML.Tag.BLOCKQUOTE.toString());
<a name="l00330"></a>00330             desiredTags.add(HTML.Tag.CITE.toString());
<a name="l00331"></a>00331             desiredTags.add(HTML.Tag.CODE.toString());
<a name="l00332"></a>00332             desiredTags.add(HTML.Tag.EM.toString());
<a name="l00333"></a>00333             desiredTags.add(HTML.Tag.H4.toString());
<a name="l00334"></a>00334             desiredTags.add(HTML.Tag.I.toString());
<a name="l00335"></a>00335             desiredTags.add(HTML.Tag.KBD.toString());
<a name="l00336"></a>00336             desiredTags.add(HTML.Tag.SAMP.toString());
<a name="l00337"></a>00337             desiredTags.add(HTML.Tag.SMALL.toString());
<a name="l00338"></a>00338             desiredTags.add(HTML.Tag.STRONG.toString());
<a name="l00339"></a>00339             desiredTags.add(HTML.Tag.SUB.toString());
<a name="l00340"></a>00340             desiredTags.add(HTML.Tag.SUP.toString());
<a name="l00341"></a>00341             desiredTags.add(HTML.Tag.TT.toString());
<a name="l00342"></a>00342             desiredTags.add(HTML.Tag.VAR.toString());
<a name="l00343"></a>00343             desiredTags.add(HTML.Tag.S.toString());
<a name="l00344"></a>00344             desiredTags.add(HTML.Tag.U.toString());
<a name="l00345"></a>00345             desiredTags.add(HTML.Tag.DIV.toString());
<a name="l00346"></a>00346             desiredTags.add(HTML.Tag.PRE.toString());
<a name="l00347"></a>00347             desiredTags.add(HTML.Tag.BR.toString());
<a name="l00348"></a>00348             desiredTags.add(HTML.Tag.TITLE.toString());
<a name="l00349"></a>00349             <span class="comment">//Si el tag es deseado retorno true sino false</span>
<a name="l00350"></a>00350             <span class="keywordflow">if</span> (desiredTags.contains(tag)) {
<a name="l00351"></a>00351                 <span class="keywordflow">return</span> <span class="keyword">true</span>;
<a name="l00352"></a>00352             }
<a name="l00353"></a>00353             <span class="keywordflow">return</span> <span class="keyword">false</span>;
<a name="l00354"></a>00354         }
<a name="l00355"></a>00355 
<a name="l00356"></a>00356         @Override
<a name="l00357"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider_1_1_parser.html#a1a351dca2ed508a436546687e782d329">00357</a>         <span class="keyword">public</span> <span class="keywordtype">void</span> <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider_1_1_parser.html#a1a351dca2ed508a436546687e782d329">handleStartTag</a>(HTML.Tag t, MutableAttributeSet a, <span class="keywordtype">int</span> pos) {
<a name="l00358"></a>00358             <span class="comment">// Empiezo el analisis de los tags</span>
<a name="l00359"></a>00359             <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider_1_1_parser.html#ae14a442ac7232170e7db1e8f872d6f8a">handleSimpleTag</a>(t, a, pos);
<a name="l00360"></a>00360         }
<a name="l00361"></a>00361 
<a name="l00367"></a>00367         @Override
<a name="l00368"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider_1_1_parser.html#a293631e270d69e91a29de14c980bc9c8">00368</a>         <span class="keyword">public</span> <span class="keywordtype">void</span> <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider_1_1_parser.html#a293631e270d69e91a29de14c980bc9c8">handleText</a>(<span class="keywordtype">char</span>[] data, <span class="keywordtype">int</span> pos) {
<a name="l00369"></a>00369             <span class="keywordflow">if</span> (isUndesiredTag == <span class="keyword">true</span>) <span class="comment">//Si el tag es indeseado no es texto</span>
<a name="l00370"></a>00370             <span class="comment">//que desee leer</span>
<a name="l00371"></a>00371             {
<a name="l00372"></a>00372                 isUndesiredTag = <span class="keyword">false</span>;
<a name="l00373"></a>00373                 <span class="keywordflow">return</span>;
<a name="l00374"></a>00374             }
<a name="l00375"></a>00375             <span class="keywordflow">if</span> (String.valueOf(data).trim().isEmpty()) <span class="comment">//Si el tag esta vacio de texto</span>
<a name="l00376"></a>00376             <span class="comment">//No almaceno y retorno</span>
<a name="l00377"></a>00377             {
<a name="l00378"></a>00378 
<a name="l00379"></a>00379                 <span class="keywordflow">return</span>;
<a name="l00380"></a>00380             }
<a name="l00381"></a>00381             <span class="keywordflow">if</span> (permisible) {
<a name="l00382"></a>00382                 <span class="comment">//Cada tag que contiene texto es almacenado</span>
<a name="l00383"></a>00383                 <span class="comment">//En el string builder. Dicho Texto de cada tag</span>
<a name="l00384"></a>00384                 <span class="comment">//Luego es separado por un patron de espacio _ espacio</span>
<a name="l00385"></a>00385                 <span class="comment">//Para luego poder ser analizado con mas facilidad por el</span>
<a name="l00386"></a>00386                 <span class="comment">//Search engine</span>
<a name="l00387"></a>00387                 SBConcatText.append(String.valueOf(data).toLowerCase().trim());
<a name="l00388"></a>00388                 SBConcatText.append(<span class="stringliteral">&quot;_&quot;</span>);
<a name="l00389"></a>00389             }
<a name="l00390"></a>00390         }
<a name="l00391"></a>00391 
<a name="l00397"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider_1_1_parser.html#aa7bd808ea40da16b1e666113db1b1616">00397</a>         <span class="keyword">protected</span> <span class="keywordtype">void</span> <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider_1_1_parser.html#aa7bd808ea40da16b1e666113db1b1616">handleLink</a>(URL <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider_1_1_parser.html#aaac5fcd0aa0bdd67e4c6842d2657231d">base</a>, String str) {
<a name="l00398"></a>00398             <span class="keywordflow">try</span> {
<a name="l00399"></a>00399                 URL url = <span class="keyword">new</span> URL(base, str);
<a name="l00400"></a>00400                 <span class="comment">//si encuentra una url la agrega a un contenedor de url,</span>
<a name="l00401"></a>00401                 <span class="comment">//verifica la url q crea y sino te tira un</span>
<a name="l00402"></a>00402                 <span class="comment">//exception</span>
<a name="l00403"></a>00403                 <span class="keywordflow">if</span> (<a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a234dbec15186b1bc7750700697ef233b">report</a>.<a class="code" href="interfacecom_1_1spider_1_1jspiderlibrary2_1_1_i_spider_reportable.html#a10fe4e486e6bd29efaa481e7532a84e3">spiderFoundURL</a>(base, url)) {
<a name="l00404"></a>00404                     <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a6c7061bb6b58bdbe6be1a162cf470c85">addURL</a>(url);
<a name="l00405"></a>00405                 }
<a name="l00406"></a>00406             } <span class="keywordflow">catch</span> (MalformedURLException e) {
<a name="l00407"></a>00407                 <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a953b39d3048cc57e3b5a6050a52931fa">log</a>(<span class="stringliteral">&quot;malformed URL: &quot;</span> + str );
<a name="l00408"></a>00408             }
<a name="l00409"></a>00409         }
<a name="l00410"></a>00410     }
<a name="l00411"></a>00411 
<a name="l00419"></a><a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a953b39d3048cc57e3b5a6050a52931fa">00419</a>     <span class="keyword">public</span> <span class="keywordtype">void</span> <a class="code" href="classcom_1_1spider_1_1jspiderlibrary2_1_1_spider.html#a953b39d3048cc57e3b5a6050a52931fa">log</a>(String entry) {
<a name="l00420"></a>00420         System.out.println((<span class="keyword">new</span> Date()) + <span class="stringliteral">&quot;:&quot;</span> + entry);
<a name="l00421"></a>00421     }
<a name="l00422"></a>00422 
<a name="l00423"></a>00423 }
</pre></div></div><!-- contents -->
<!-- window showing the filter options -->
<div id="MSearchSelectWindow"
     onmouseover="return searchBox.OnSearchSelectShow()"
     onmouseout="return searchBox.OnSearchSelectHide()"
     onkeydown="return searchBox.OnSearchSelectKey(event)">
<a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(0)"><span class="SelectionMark">&#160;</span>All</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(1)"><span class="SelectionMark">&#160;</span>Classes</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(2)"><span class="SelectionMark">&#160;</span>Namespaces</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(3)"><span class="SelectionMark">&#160;</span>Files</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(4)"><span class="SelectionMark">&#160;</span>Functions</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(5)"><span class="SelectionMark">&#160;</span>Variables</a></div>

<!-- iframe showing the search results (closed by default) -->
<div id="MSearchResultsWindow">
<iframe src="javascript:void(0)" frameborder="0" 
        name="MSearchResults" id="MSearchResults">
</iframe>
</div>



<hr class="footer"/><address class="footer"><small>
Generated on Thu Feb 16 2012 02:56:13 for JSpider Library by &#160;<a href="http://www.doxygen.org/index.html">
<img class="footer" src="doxygen.png" alt="doxygen"/>
</a> 1.7.6.1
</small></address>

</body>
</html>
